import requests
from lxml import etree
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt


plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签
plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示正负号


# 下载数据
def load_data():
    # 构筑五个装数据的容器
    bookType_list = []  # 书本类型
    wordCount_list = []  # 字数
    author_list = []  # 作者
    updateTime_list = []  # 更新的时间
    status_list = []  # 更新状态
    for index in range(1, 335):  # 共334页
        # 获取数据
        page_text = requests.get('https://www.17k.com/all/book/2_0_0_0_0_0_0_0_{}.html'.format(index)).text
        with open('./class/lecture%s.html'%index,'w',encoding='utf-8') as fp:
            fp.write(page_text)
        tree = etree.HTML(page_text)  # 生成一个etree对象
        tb_list = tree.xpath('/html/body/div[4]/div[3]/div[2]/table/tbody')  # 使用xpath进行数据解析
        for tb in tb_list:
            bookType_list += tb.xpath('.//td[2]/a/text()')
            wordCount_list += tb.xpath('.//td[5]/text()')
            author_list += tb.xpath('.//td[6]/a/text()')
            updateTime_list += tb.xpath('.//td[7]/text()')
            status = [txt.strip('\n ') for txt in tb.xpath('.//td[8]/em/text()')]  # 去除数据中的换行和空格
            status_list += status
    # 将数据返回主逻辑
    return bookType_list, wordCount_list, author_list, updateTime_list, status_list



def data_processing(bookType_list, wordCount_list, author_list, updateTime_list, status_list):
    # 表头
    table_name = ['书本类型', '字数', '作者', '更新日期', '更新状态']
    # 数据
    data = [
        np.array(bookType_list),
        np.array(wordCount_list),
        np.array(author_list),
        np.array(updateTime_list),
        np.array(status_list)
    ]
    # 将表头和数据拼接在一起
    data = dict(zip(table_name, data))
    # 将所有数据加载入
    df = pd.DataFrame(data)
    # 将代码存储至
    df.to_excel('./dataset/data.xlsx')
    print(df.head())
    # 对数据进行归类统计
    # 取类型前五的书本类型画出柱状图
    bookType = df['书本类型'].value_counts().index[:5]
    typeCount = df['书本类型'].value_counts().values[:5]
    bar(bookType, typeCount)
    # 画饼图
    status = df['更新状态'].value_counts().index
    print(status)
    status_count = np.array(df['更新状态'].value_counts().values)
    print(status_count)
    x = [i / sum(status_count) for i in status_count]
    print(x)
    pie(x, labels=status)


# 画饼图
def pie(x, labels):
    plt.pie(labels=labels, x=x)
    plt.show()


# 画柱状图
def bar(x, labels):
    plt.barh(x, labels, height=0.7, color='q', alpha=0.8)
    plt.show()


# 主逻辑
if __name__ == '__main__':
    bookType_lib, wordCount_lib, author_lib, updateTime_lib, status_lib = load_data()
    data_processing(bookType_lib, wordCount_lib, author_lib, updateTime_lib, status_lib)
